/*
 * @Author: wwssaabb
 * @Date: 2021-07-07 08:24:41
 * @LastEditTime: 2021-07-19 16:18:23
 * @FilePath: \demo\crawler_demo\电影天堂_demo\app.js
 */

//robots
/* User-agent: * 
Disallow: /member
Disallow: /plus/feedback_js.php
Disallow: /plus/mytag_js.php
Disallow: /plus/rss.php
Disallow: /plus/search.php
Disallow: /dede/
Disallow: /plus/recommend.php
Disallow: /plus/stow.php
Disallow: /plus/count.php
Disallow: /dededy/
Disallow: /js2/
Disallow: /gg/
Disallow: /g/ */


const axios=require('axios')
const iconv=require('iconv-lite')
const cheerio=require('cheerio')
const fs=require('fs')


let httpUrl='https://www.dytt8.net/index.htm'
let baseurl='https://www.dytt8.net'


//先获取首页数据，并处理
async function main(){  //主体方法
  const log={
    currentListIndex:0,
    currentInnerList_Index:0
  }

  if(false){
  let index=await getHTML(httpUrl)

  //处理首页文本中的class，将co_content3改为co_content8
  index=replaceText(index,'co_content3','co_content8')
  //cheerio挂载到首页
  let $=cheerio.load(index)
  let index_arr=[]  //用来装首页模块的数据

  $('.co_area2').each(function(){
    //分成两种数据来进行获取，class co_content8 有更多链接、发布时间获取
    if($('div',this).last().attr('class')==='co_content8'){
      let more_href=baseurl+$('.title_all a',this).attr('href')
      $('.title_all p em',this).remove()
      let list=[]
      let title=$('.title_all p',this).text()
      $('.co_content8 tr',this).each(function(){
        let tag,name,href
        $('td.inddline',this).each(function(i){
          if(i===1||$('a',this).attr('href')=='/app.html') return 
          $('a',this).each(function(i){
            console.log(i)
            if(i===0){
              tag=$(this).text()
            }else if(i===1){
              name=$(this).text()
              href=baseurl+$(this).attr('href')
            }
          })
          pubdate=$(this).next().text()
          list.push({tag,name,href,pubdate})
        })
        
      })
      index_arr.push({title,list,more_href})
    }else{
      let title=$('.title_all p',this).text()
      let list=[]
      $('ul a',this).each(function(){
        let name=$(this).text()
        let href=baseurl+$(this).attr('href')
        list.push({name,href})
      })
      index_arr.push({title,list})
    }
  })

  //console.log(index_arr)
  fs.writeFileSync('./json/index.json',JSON.stringify(index_arr)) //将获取到的首页数据保存到json文件夹中

  }   

  
  //重置$
  $=null
  //获取国内电影、日韩、欧美、精品
  if(false){
  let chain_films={name:'国内电影',url:'https://www.dytt8.net/html/gndy/china/index.html',baseurl:'https://www.dytt8.net/html/gndy/china/',list:[]} 
  let jk_films={name:'日韩电影',url:'https://www.dytt8.net/html/gndy/rihan/index.html',baseurl:'https://www.dytt8.net/html/gndy/rihan/',list:[]}
  let ea_films={name:'欧美电影',url:'https://www.dytt8.net/html/gndy/oumei/index.html',baseurl:'https://www.dytt8.net/html/gndy/oumei/',list:[]}
  let boutique_films={name:'精品电影',url:'https://www.dytt8.net/html/gndy/dyzz/index.html',baseurl:'https://www.dytt8.net/html/gndy/dyzz/',list:[]}

  let action_arr=[chain_films,jk_films,ea_films,boutique_films]
  }
  let action_arr=require('./json/preview8.json')

  function getPromiseAll(item,index) {
    return new Promise(async (resolve)=>{
      
      if(false){
        let html=await getHTML(item.url)
        let $=cheerio.load(html)
        getFilms(html,index)
        //获取剩余的页数地址
        action_arr[index].less_pages=[]
        $('.x select option').each(function(){
          action_arr[index].less_pages.push(item.baseurl+$(this).attr('value'))
        })
        //爬取剩余的电影，数量有点大,访问间隔5000ms，//正式运行记得删掉下面的slice
      await Promise.all(action_arr[index].less_pages.map(async (i,delay)=>{
        return new Promise(async (resolve)=>{
          if(!action_arr[index].less_pages[delay].isDone){
            setTimeout(async ()=>{
              console.log('现在在获取 '+action_arr[index].name+'，已获取到'+i.match(/(?<=\_)\d+(?=\.)/g)[0]+'页')
              if(!action_arr[index].less_pages[delay].isDone){
                let html=await getHTML(i)
                getFilms(html,index)
                fs.writeFileSync('./json/preview4.json',JSON.stringify(action_arr))
              }
              resolve()
            },delay*2000)
          }else{
            resolve()
          }
        })
      }))

    }
      //fs.writeFileSync('./json/preview4.json',JSON.stringify(action_arr))

      console.log(action_arr[index].list.length)
      let skipArr=[796,2579,2730,3225]
      //爬取list里面的电影数据，info信息、cover海报、publishDate发布时间
      let info_promiseArr=[]  //正式运行记得删掉下面的slice
      await Promise.all(action_arr[index].list.map(async (item,_index)=>{
        if(action_arr[index].list[_index].isDone||skipArr.includes(_index)) return Promise.resolve()
        if(_index===0){
          info_promiseArr[_index]=new Promise(async resolve=>{
            setTimeout(async ()=>{
              if(!action_arr[index].list[_index].isDone){
              let info_page=await getHTML(action_arr[index].list[_index].href)
              console.log(action_arr[index].list[_index].href)
              console.log('list:'+action_arr[index].name,',index:'+_index+'/'+action_arr[index].list.length)
              let $=cheerio.load(info_page)
              action_arr[index].list[_index].info=getInfo($('#Zoom').html())
              action_arr[index].list[_index].download_url=$('#Zoom span a').attr('href')
              action_arr[index].list[_index].cover=$('#Zoom img').attr('src')
              action_arr[index].list[_index].isDone=true
              fs.writeFileSync('./json/preview8.json',JSON.stringify(action_arr))
              }
              resolve()
            },2000)
          })
          return await info_promiseArr[_index]
        }else{
          info_promiseArr[_index]=new Promise(async resolve=>{
            await info_promiseArr[_index-1]
            await new Promise(resolve=>{
              setTimeout(async ()=>{
                if(!action_arr[index].list[_index].isDone){
                  let info_page=''
                  try {
                    info_page=await getHTML(action_arr[index].list[_index].href)
                  } catch (error) {
                    console.log('连接错误，等待5s重连')
                    await new Promise(async resolve=>{
                      info_page=await getHTML(action_arr[index].list[_index].href)
                      resolve()
                    },5000)
                  }
                  console.log(action_arr[index].list[_index].href)
                  console.log('list:'+action_arr[index].name,',index:'+_index+'/'+action_arr[index].list.length)
                  let $=cheerio.load(info_page)
                  action_arr[index].list[_index].info=getInfo($('#Zoom').html())
                  action_arr[index].list[_index].download_url=$('#Zoom span a').attr('href')
                  action_arr[index].list[_index].cover=$('#Zoom img').attr('src')
                  action_arr[index].list[_index].isDone=true
                  fs.writeFileSync('./json/preview8.json',JSON.stringify(action_arr))
                }
                resolve()
              },2000)
            }) 
            resolve()
          })
          return info_promiseArr[_index]
        }
      }))
      resolve()
    })
    
  }

  function getFilms(html,index){
    let $=cheerio.load(html)
    $('.co_content8 .tbspan').each(function(){
      let obj={}
      $('b a.ulink',this).each(function(i){
        if(i==1||index==3){
          obj.name=$(this).text()
          obj.href=baseurl+$(this).attr('href')
        }
      })
      obj.isDone=true
      action_arr[index].list.push(obj)
    })
  }

  //爬取页数获取电影
  await Promise.all(action_arr.map(async (item,index)=>{
    if(index===0){
      action_arr[index].promiseAll=getPromiseAll(item,index)
      return await action_arr[index].promiseAll
    }else{
      return Promise.resolve()
      action_arr[index].promiseAll=new Promise(async resolve=>{
        await action_arr[index-1].promiseAll
        await getPromiseAll(item,index)
        resolve()
      })
      return action_arr[index].promiseAll
    }
  }))

  console.log('main') //
  
  //后续代码处理
  /* let filmAll=[]
  action_arr.forEach((item,index)=>{
    action_arr[index].remove('promise')
    delete action_arr[index].less_pages
    filmAll.push(...action_arr[index].list)
  })
 */

  //fs.writeFileSync('./json/preview4.json',JSON.stringify(action_arr))

  
}

main()


async function getHTML(http) {  //获取html
  let {data}=await axios(http,{responseType:'arraybuffer'})
  //let res=await axios(http,{responseType:'arraybuffer'})
  let html=iconv.decode(data,'gb2312')
  //fs.writeFileSync('./html/index.html',html)
  return html
}
function replaceText(context,str,text){   //替换文字
  //let context=fs.readFileSync(path,'utf8')
  if(Array.isArray(str)){
    str.forEach(i=>{
      context=context.replace(new RegExp(i,'g'),text)
    })
  }else{
    context=context.replace(new RegExp(str,'g'),text)
  }
  //fs.writeFileSync('./html/index2.html',context)
  return context
}

/* function getInfo(context){  //只需要传入' #Zoom span'的文本，返回info对象
  let info = context.match(/\<br\>\◎?.[^\<]+\<br\>/g)
  if(info==null){
    info = context.match(/\<div\>\◎?.[^\<]+\<div\>/g)
  }
  info = info.join()
  info = info.replace(/\<br\>/g, '').replace(/\&nbsp\;/g,' ').split('◎')
  let obj={}
  info.forEach(i => {
    if(i!=''){
      let key,value
      if(i.indexOf('　')!==-1){
        key=i.replace(/　{2,2}|^　/g,'').replace(/(?<!\w)\,/,'　　').slice(0,i.indexOf('　')+1).trim()
        value=i.replace(/　{2,2}|^　/g,'').replace(/(?<!\w)\,/,'　　').slice(i.indexOf('　')+1).trim().replace(/(　| )/g,'')
      }else{
        key=i.slice(0,i.indexOf(' ')+1).trim()
        value=i.slice(i.indexOf(' ')+1).trim().replace(' ','')
      }
      value[value.length-1]===','?value=value.slice(0,value.length-1):null
      key==='标签'?value=value.split('|'):null
      obj[key]=value
    }
  })
  return obj
} */

function getInfo(context){  //只需要传入' #Zoom span'的文本，返回info对象
  let info = context.replace(/&nbsp;/g,' ').replace(/\<img.[^\>]+\>/g,'').match(/(?<=\<br\>)◎?.[^\<]+(?=\<br\>)/g)
  if(info==null){
    info = context.replace(/&nbsp;/g,' ').replace(/\<img.[^\>]+\>/g,'').match(/(?<=\<div\>)◎?.[^\<]+(?=\<\/div\>)/g)
  }
  info = info.join()
  info = info.replace(/\<br\>/g, '').replace(/\<\/?div\>/g, '').replace(/\&nbsp\;/g,' ').split('◎')
  let obj={}
  info.forEach(i => {
    if(i!=''&&i!=' ,'&&i!=',,'){
      let key,value
      if(i.indexOf('　')!==-1){
        key=i.replace(/　{2,2}|^　/g,'').replace(/(?<!\w)\,/,'　　').slice(0,i.indexOf('　')+1).trim().replace(/\s/g,'')
        value=i.replace(/　{2,2}|^　/g,'').replace(/(?<!\w)\,/,'　　').slice(i.indexOf('　')+1).trim().replace(/(　| )/g,'')
      }else{
        key=i.slice(0,i.indexOf(' ')+1).trim().replace(/\s/g,'')
        value=i.slice(i.indexOf(' ')+1).trim()
      }
      value[value.length-1]===','?value=value.slice(0,value.length-1):null
      key==='标签'?value=value.split('|'):null
      obj[key]=value
    }
  })
  return obj
}